//////////////////////////////////////////
//
// OpenCL host program template
//
//                   
// Created by dongwang@2016.01.10
//
/////////////////////////////////////////
#define CL_HPP_CL_1_2_DEFAULT_BUILD
#define CL_HPP_TARGET_OPENCL_VERSION 120
#define CL_HPP_MINIMUM_OPENCL_VERSION 120
#define CL_HPP_ENABLE_PROGRAM_CONSTRUCTION_FROM_ARRAY_COMPATIBILITY 1
#define CL_USE_DEPRECATED_OPENCL_1_2_APIS

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <CL/opencl.h>
#include "../device/custypedef.h"

// user defined library
#include "ocl_util.h"
#include "timer.h"

using namespace ocl_util;

//----------- Design Parameters --------------//
// select what platform is used
const char *vendor_name = "Xilinx";
//const char *vendor_name = "AMD";
//const char *vendor_name = "Intel"; // SDK version <= 19.1
//#if defined(SW_EMU) // SDK version >= 19.3
//const char *vendor_name = "Intel(R) FPGA Emulation Platform for OpenCL(TM)";
//#else
//const char *vendor_name = "Intel(R) FPGA SDK for OpenCL(TM)";
//#endif
#define DEVICE_TYPE CL_DEVICE_TYPE_ACCELERATOR
//#define DEVICE_TYPE CL_DEVICE_TYPE_GPU

// Define the binary name
const char *kernel_name_Rd = "featureRead";
const char *kernel_name_Pr = "paramRead";
const char *kernel_name_Wr = "memWrite";




//------------ Global Functions & Variables ------------//
cl_uint num_devices = 0;
cl_platform_id platform_id = NULL;
cl_context context = NULL;
cl_program program = NULL;
scoped_array<cl_device_id> device;
scoped_array<cl_kernel> kernel;
scoped_array<cl_kernel> kernel2;
scoped_array<cl_kernel> kernel3;
scoped_array<cl_command_queue> queue_Rd;
scoped_array<cl_command_queue> queue_Pr;
scoped_array<cl_command_queue> queue_Wr;
scoped_array<cl_mem> input_a_buf;
scoped_array<cl_mem> input_b_buf;
scoped_array<cl_mem> output_c_buf;
scoped_array<DATA_TYPE *> input_a;
scoped_array<DATA_TYPE *> input_b;
scoped_array<DATA_TYPE *> output_c;
scoped_array<DATA_TYPE *> output_cCPU;

void cleanup();
void conv2d(scoped_array<DATA_TYPE*>& img, scoped_array<DATA_TYPE*>& knl, scoped_array<DATA_TYPE*>& RSL);

int main(int argc, char** argv)
{
	cl_int status;
	unsigned int num_err;               // number of error results returned
    unsigned int device_ptr;

	Timer t;                            // Timer used for performance measurement
	
	
	Timer t_hw, t_sw;                            // Timer used for performance measurement
	
	if (argc != 2){
	printf("Error: wrong commad format, usage:\n");  
	printf("%s <inputfile>\n", argv[0]);
	return EXIT_FAILURE;
	}
	
	printf("********************************\n");
	printf("Vector Add/Mult Benchmark\n");
	printf("********************************\n");
    printf("Data Size for each device: %d MBytes\n", ((unsigned int)(sizeof(float)*DATA_SIZE)/(1024*1024)));
	
	// Connect to the desired platform
	platform_id = findPlatform(vendor_name);
	if(platform_id == NULL) {
		printf("ERROR: Unable to find the desired OpenCL platform.\n");
		return EXIT_FAILURE;
	}

	// Query the available OpenCL device
	device.reset(getDevices(platform_id, DEVICE_TYPE, &num_devices));
	printf("Platform: %s\n", getPlatformName(platform_id).c_str());
    printf("Totally %d device(s) are found\n", num_devices);
    device_ptr = 0; // only use one device, select the proper idx
    num_devices = 1; // reset the num of device to 1
	//for(unsigned i = 0; i < num_devices; ++i) {
		printf("  Device %d: %s\n", device_ptr, getDeviceName(device[device_ptr]).c_str());
		displayDeviceInfo(device[device_ptr]);
	//}
	
	// Initialize the data set for each device
	input_a.reset(num_devices);
	input_b.reset(num_devices);
	output_c.reset(num_devices);
	output_cCPU.reset(num_devices);
	for(unsigned i = 0; i < num_devices; ++i) {

		input_a[i] = (DATA_TYPE *)alignedMalloc(sizeof(DATA_TYPE)*DATA_SIZE, DMA_ALIGNMENT);
		input_b[i] = (DATA_TYPE *)alignedMalloc(sizeof(DATA_TYPE)*(KERNEL_SIZE), DMA_ALIGNMENT);
		output_c[i] = (DATA_TYPE *)alignedMalloc(sizeof(DATA_TYPE)*OUTPUT_SIZE, DMA_ALIGNMENT);
		output_cCPU[i] = (DATA_TYPE *)alignedMalloc(sizeof(DATA_TYPE)*OUTPUT_SIZE, DMA_ALIGNMENT);
		if(input_a[i] == NULL || input_b[i] == NULL || output_c[i] == NULL)
		{
			printf("Not enough memory !!!");
			alignedFree(input_a[i]);
			alignedFree(input_b[i]);
			alignedFree(output_c[i]);
			alignedFree(output_cCPU[i]);
			return EXIT_FAILURE;
		}
		
		for(unsigned int j = 0; j < DATA_SIZE; j++) {
			input_a[i][j] = DATA_TYPE(j%16);
		}
		for(unsigned int j = 0; j < KERNEL_SIZE; j++) {
			input_b[i][j] = DATA_TYPE(j%16);
		}
		
		memset(output_c[i], 0, sizeof(DATA_TYPE)*OUTPUT_SIZE);
		memset(output_cCPU[i], 0, sizeof(DATA_TYPE)*OUTPUT_SIZE);
	}

	// Create the context.
	context = clCreateContext(NULL, num_devices, device, NULL, NULL, &status);
	checkError(status, "Failed to create context");

	// Create Program Objects
	char *kernel_file_name=argv[1];

	// Create the program for all device. All devices execute the same kernel.
	program = createProgramFromFile(context, (const char *) kernel_file_name, device, num_devices);
	
	// Create per-device objects.
	queue_Rd.reset(num_devices);
	queue_Pr.reset(num_devices);
	queue_Wr.reset(num_devices);
	kernel.reset(num_devices);
	kernel2.reset(num_devices);
	kernel3.reset(num_devices);
	input_a_buf.reset(num_devices);
	input_b_buf.reset(num_devices);
	output_c_buf.reset(num_devices);
	
	for(unsigned i = 0; i < num_devices; ++i) {
		// Command queue
		queue_Rd[i] = clCreateCommandQueue(context, device[i], CL_QUEUE_PROFILING_ENABLE, &status);
		checkError(status, "Failed to create command queue_Rd");

		queue_Pr[i] = clCreateCommandQueue(context, device[i], CL_QUEUE_PROFILING_ENABLE, &status);
		checkError(status, "Failed to create command queue_Pr");

		queue_Wr[i] = clCreateCommandQueue(context, device[i], CL_QUEUE_PROFILING_ENABLE, &status);
		checkError(status, "Failed to create command queue_Wr");
		
		// Kernel
		kernel[i] = clCreateKernel(program, kernel_name_Rd, &status);
		checkError(status, "Failed to create kernel");
		kernel2[i] = clCreateKernel(program, kernel_name_Wr, &status);
		checkError(status, "Failed to create kernel");
		kernel3[i] = clCreateKernel(program, kernel_name_Pr, &status);
		checkError(status, "Failed to create kernel");
		
		// Input buffers
		input_a_buf[i] = clCreateBuffer(context, CL_MEM_READ_ONLY, 
			DATA_SIZE * sizeof(DATA_TYPE), NULL, &status);
		checkError(status, "Failed to create buffer for input A");

		// Input buffers
		input_b_buf[i] = clCreateBuffer(context, CL_MEM_READ_ONLY, 
			(KERNEL_SIZE) * sizeof(DATA_TYPE), NULL, &status);
		checkError(status, "Failed to create buffer for input A");
		
		// Output buffers
		output_c_buf[i] = clCreateBuffer(context, CL_MEM_WRITE_ONLY, 
			OUTPUT_SIZE * sizeof(DATA_TYPE), NULL, &status);
		checkError(status, "Failed to create buffer for output B");

		// Enque input buffers, blocking write is used
		status = clEnqueueWriteBuffer(queue_Rd[i], input_a_buf[i], CL_TRUE,
			0, DATA_SIZE * sizeof(DATA_TYPE), input_a[i], 0, NULL, NULL);
		checkError(status, "Failed to transfer input A");
		
		// Enque input buffers, blocking write is used
		status = clEnqueueWriteBuffer(queue_Pr[i], input_b_buf[i], CL_TRUE,
			0, (KERNEL_SIZE) * sizeof(DATA_TYPE), input_b[i], 0, NULL, NULL);
		checkError(status, "Failed to transfer input A");
		
		// Set kernel arguments.
		unsigned argi = 0;
        unsigned data_num;
        // if(DATA_SIZE%VEC_SIZE!=0){
        //     printf("Error: incorrect setting of DATA_SIZE or VEC_SIZE !!!\n\n");
        //     return EXIT_FAILURE;
        // }
        // else
            data_num = DATA_SIZE/VEC_SIZE;
		
		status = clSetKernelArg(kernel[i], argi++, sizeof(cl_mem), &input_a_buf[i]);
		checkError(status, "Failed to set argument %d", argi - 1);
		
		status = clSetKernelArg(kernel[i], argi++, sizeof(cl_uint), &data_num);
		checkError(status, "Failed to set argument %d", argi - 1);
		
		argi = 0;
				
		status = clSetKernelArg(kernel2[i], argi++, sizeof(cl_mem), &output_c_buf[i]);
		checkError(status, "Failed to set argument %d", argi - 1);

		argi = 0;
				
		status = clSetKernelArg(kernel3[i], argi++, sizeof(cl_mem), &input_b_buf[i]);
		checkError(status, "Failed to set argument %d", argi - 1);

		status = clSetKernelArg(kernel3[i], argi++, sizeof(cl_uint), &data_num);
		checkError(status, "Failed to set argument %d", argi - 1);
	}
	printf("start recording ...\n");
	// Recorde the start time
	
	
	// Execute the kernel
	scoped_array<cl_event> kernel_event(num_devices);
	scoped_array<cl_event> kernel_event2(num_devices);
	scoped_array<cl_event> kernel_event3(num_devices);
	scoped_array<cl_event> readevent(num_devices);
	t.start();
	for(unsigned i = 0; i < num_devices; ++i) {
		status = clEnqueueTask(queue_Pr[i], kernel3[i], 0, NULL, &kernel_event3[i]);
		checkError(status, "Failed to launch kernel");

		status = clEnqueueTask(queue_Rd[i], kernel[i], 0, NULL, &kernel_event[i]);
		checkError(status, "Failed to launch kernel");
		
		status = clEnqueueTask(queue_Wr[i], kernel2[i], 0, NULL, &kernel_event2[i]);
		checkError(status, "Failed to launch kernel");
	}
	
	// Wait for all kernel to finishs
	clWaitForEvents(num_devices, kernel_event2);

	//Recorde the end time
	t.stop();
	float time = t.get_time_s();
	
	// Read back the results from the device to verify the output
	//
	for(unsigned i = 0; i < num_devices; ++i) {

		status = clEnqueueReadBuffer(queue_Wr[i], output_c_buf[i], CL_FALSE, 
			0, sizeof(DATA_TYPE) * OUTPUT_SIZE, output_c[i], 0, NULL, &readevent[i]);  
		checkError(status, "Failed to set transfer output B");
	}

	// Wait for reads to finish
	clWaitForEvents(num_devices, readevent);

	// Evaluate the performance
	// average bandwidth
	float bw = DATA_SIZE * sizeof(float) / (time * 1000000.0f);
	printf("\nPerformance Summary\n");
	printf("Total runtime: %fs \n", time);
	printf("Average device2ddr bandwidth (RD or WR): %f MBytes/s\n", bw);
	// bandwidth for each device
	for(unsigned i = 0; i < num_devices; ++i) {
		cl_ulong time_ns = getKernelStartEndTime(kernel_event[i]);
		printf("Kernel runtime (device %d): %0.3f ms\n\n", i, double(time_ns) * 1e-6);
	}
	
	// Validate the results
	num_err = 0;
	// for(unsigned i = 0; i < num_devices; ++i) {
	// 	for (unsigned int j = 0; j < DATA_SIZE; j++) 
	// 		if (output_c[i][j] != input_a[i][j]+input_b[i][j] && num_err<10){
	// 			printf("Error: num=%d, input_a=%f, input_b=%f, output=%f\n", j, float(input_a[i][j]), float(input_b[i][j]), float(output_c[i][j]));
	// 			num_err++; // count how many error are found
	// 	}
	// }
	
	// Release resource
	t.start();
	
	conv2d(input_a,input_b,output_cCPU);
	t.stop();
	time = t.get_time_s();
	printf("CPU execute time: %fs \n", time);
	for(unsigned i = 0; i < num_devices; ++i) {
		for (unsigned int j = 0; j < OUTPUT_SIZE; j++){
			if (output_c[i][j] != output_cCPU[i][j] && num_err<65){
				printf("Error: num=%d, output=%f, outputCPU=%f\n", j, float(output_c[i][j]), float(output_cCPU[i][j]));
				num_err++; // count how many error are found
			}
		}
	}
	cleanup();
	
	if(num_err == 0){
		printf("\nTest passed!\n");
		return EXIT_SUCCESS;
	}
	else{
		printf("\nTest failed!\n");
		printf("Summary: '%d/%d' error values!\n", num_err, OUTPUT_SIZE);
		return EXIT_FAILURE;
	}
}



void conv2d(scoped_array<DATA_TYPE*>& img, scoped_array<DATA_TYPE*>& knl, scoped_array<DATA_TYPE*>& RSL){
	float tmp = 0;
	for(int i = 0; i < DATA_SIZE_W - DATA_SIZE_K + 1; i++){
		for(int j = 0; j < DATA_SIZE_H - DATA_SIZE_K + 1; j++){
			for(int m = 0; m < DATA_SIZE_M; m++){
				tmp = 0;
				int img_offset_step = DATA_SIZE_W * DATA_SIZE_H * m + DATA_SIZE_W * j + i;
				for(int kw = 0; kw < DATA_SIZE_K; kw++){
					for(int kh = 0; kh < DATA_SIZE_K; kh++){
						for(int n = 0; n < DATA_SIZE_N; n++){
							int img_offset = img_offset_step + kh * DATA_SIZE_W + kw;
							int knl_offset = n * DATA_SIZE_K * DATA_SIZE_K + kh * DATA_SIZE_K + kw;
							tmp += img[0][img_offset] * knl[0][knl_offset];
						}
					}
				}
				int RSL_offset = i + (DATA_SIZE_W - DATA_SIZE_K + 1) * j + (DATA_SIZE_W - DATA_SIZE_K + 1) * (DATA_SIZE_H - DATA_SIZE_K + 1) * m;
				RSL[0][RSL_offset] = tmp;
			}
		}
	}
	return;
}

// Release all memory resources here
void cleanup()
{

	// Release the opencl runtime resource allocated
	for(unsigned i = 0; i < num_devices; ++i) {
		if(kernel && kernel[i]) {
			clReleaseKernel(kernel[i]);
		}
		if(queue_Rd && queue_Rd[i]) {
			clReleaseCommandQueue(queue_Rd[i]);
		}
		if(queue_Pr && queue_Pr[i]) {
			clReleaseCommandQueue(queue_Pr[i]);
		}
		if(queue_Wr && queue_Wr[i]) {
			clReleaseCommandQueue(queue_Wr[i]);
		}
		if(input_a_buf && input_a_buf[i]) {
			clReleaseMemObject(input_a_buf[i]);
		}
		if(input_b_buf && input_b_buf[i]) {
			clReleaseMemObject(input_b_buf[i]);
		}
		if(output_c_buf && output_c_buf[i]) {
			clReleaseMemObject(output_c_buf[i]);
		}
	}
	
	if(program) {
		clReleaseProgram(program);
	}
	if(context) {
		clReleaseContext(context);
	}

	// Release the memory resource allocated
	for(unsigned i = 0; i < num_devices; ++i) {
		alignedFree(input_a[i]);
		alignedFree(input_b[i]);
		alignedFree(output_c[i]);
	}
}
